#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :bing_pic.py
# @Time      :2023/10/10 
# @Author    :CL
# @email     :1037654919@qq.com

import requests
import os
import re
from bs4  import BeautifulSoup
import time
from retrying import retry
from tqdm import tqdm
from util import mongo_manager
from multiprocessing import Pool
#用于维护采集图片列表的数据库，只能172.23开头的ip使用，如果不需要数据库，可以注释这一部分，但不保证图片去重
images_url = mongo_manager('images_url',db = 'cl_data')

# 保存图片的base链接，根据自己的需要修改
basepath = '/media/chenglei3/77D014CED257D1411/images/bing_pic/'
# 输入关键词和页码，输出图片
@retry(stop_max_attempt_number=3, wait_fixed=2000)
def get_bing_image(word='爆炸头', SFX=2):
    first = (SFX - 1) * 35 +1
    relp=(SFX - 1) * 35
    headers = {
        "accept": "*/*",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "priority": "u=1, i",
        "referer": "https://www.bing.com/images/search?q=%e5%81%9c%e8%bd%a6%e5%9c%ba%e5%85%a5%e5%8f%a3&form=RESTAB&first=1",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-arch": "\"x86\"",
        "sec-ch-ua-bitness": "\"64\"",
        "sec-ch-ua-full-version": "\"124.0.6367.60\"",
        "sec-ch-ua-full-version-list": "\"Chromium\";v=\"124.0.6367.60\", \"Google Chrome\";v=\"124.0.6367.60\", \"Not-A.Brand\";v=\"99.0.0.0\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-model": "\"\"",
        "sec-ch-ua-platform": "\"Linux\"",
        "sec-ch-ua-platform-version": "\"5.4.0\"",
        "sec-fetch-dest": "empty",
        "sec-fetch-mode": "cors",
        "sec-fetch-site": "same-origin",
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    }
    cookies = {
        "_IDET": "VSNoti2=20230908",
        "ipv6": "hit=1716540413166",
        "MUID": "1F9DBAD639B4600407AAA990384F6146",
        "MUIDB": "1F9DBAD639B4600407AAA990384F6146",
        "SRCHD": "AF=NOFORM",
        "SRCHUID": "V=2&GUID=EE14C4042ECC43ECAF6C9C7AD67E3550&dmnchg=1",
        "MMCASM": "ID=92A4CF809FA9426A9D91DF723D77766E",
        "imgv": "lodlg=1&gts=20230905",
        "fbar": "imgfbar=1",
        "_UR": "QS=0&TQS=0&cdxcls=0",
        "_HPVN": "CS=eyJQbiI6eyJDbiI6MywiU3QiOjAsIlFzIjowLCJQcm9kIjoiUCJ9LCJTYyI6eyJDbiI6MywiU3QiOjAsIlFzIjowLCJQcm9kIjoiSCJ9LCJReiI6eyJDbiI6MywiU3QiOjAsIlFzIjowLCJQcm9kIjoiVCJ9LCJBcCI6dHJ1ZSwiTXV0ZSI6dHJ1ZSwiTGFkIjoiMjAyMy0xMC0wOVQwMDowMDowMFoiLCJJb3RkIjowLCJHd2IiOjAsIkRmdCI6bnVsbCwiTXZzIjowLCJGbHQiOjAsIkltcCI6MTJ9",
        "MicrosoftApplicationsTelemetryDeviceId": "4aebd269-5f77-40b0-8e68-92ca225a590f",
        "_Rwho": "u=d&ts=2024-05-24",
        "SRCHUSR": "DOB=20230905&T=1716536279000&TPC=1716536284000",
        "_EDGE_S": "SID=2B2B6A47AF2F6CFD0B117ECEAE456D4C&mkt=en-us",
        "SRCHHPGUSR": "SRCHLANG=zh-Hans&BRW=XW&BRH=M&CW=1856&CH=932&SCW=1841&SCH=1720&DPR=1.0&UTC=480&DM=0&PV=5.4.0&HV=1716536811&PRVCW=1856&PRVCH=932&WTS=63852133079&IG=6ABFF1BF22C24F57BC178BC271A9416B",
        "_C_ETH": "1",
        "_RwBf": "r=0&ilt=13&ihpd=0&ispd=0&rc=35&rb=0&gb=0&rg=200&pc=30&mtu=0&rbb=0&g=0&cid=&clo=0&v=8&l=2024-05-24T07:00:00.0000000Z&lft=0001-01-01T00:00:00.0000000&aof=0&o=2&p=&c=&t=0&s=0001-01-01T00:00:00.0000000+00:00&ts=2024-05-24T07:47:12.0467562+00:00&rwred=0&wls=&wlb=&lka=0&lkt=0&TH=&ard=0001-01-01T00:00:00.0000000&rwdbt=0001-01-01T00:00:00.0000000&rwflt=0001-01-01T00:00:00.0000000&wle=&ccp=&cpt=&aad=0",
        "_SS": "SID=2B2B6A47AF2F6CFD0B117ECEAE456D4C&R=35&RB=0&GB=0&RG=200&RP=30"
    }
    url = "https://www.bing.com/images/async"
    params = {
        "q": word,
        "first":first,
        "count": "35",
        "cw": "1856",
        "ch": "932",
        "relp":  "35",
        "datsrc": "I",
        "layout": "RowBased_Landscape",
        "apc": "1",
        "imgbf": "DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAArUkdz+NfTNlhQBYYKdIOoAI1wECAWaSEIgOAl85BkGJAMJAcSADbHUlGwQRCTp4TbCwcFPLrCg4H2AAAAAAAAA==",
        "mmasync": "1",
        "dgState": "x*464_y*964_h*187_c*2_i*36_r*6",
        "IG": "6ABFF1BF22C24F57BC178BC271A9416B",
        "SFX":SFX,
        "iid": "images.5743"
    }
    response = requests.get(url, headers=headers, cookies=cookies, params=params)
    if response.status_code ==200:
        print(response.url,response)
        return response.text


@retry(wait_fixed=2000,stop_max_attempt_number=5)
def get_pic_data(url ='https://tse3.mm.bing.net/th/id/OIP.1peV1ejRbo9k1qNU2Ny58wHaEi'):
    # print('begin:',url)
    headers = {
        "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
        "Referer": "https://www.bing.com/",
        "sec-ch-ua-mobile": "?0",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
        "sec-ch-ua-platform": "\"Linux\""
    }

    params = {
        "pid": "ImgDet",
        "rs": "1"
    }
    response = requests.get(url, headers=headers, params=params,timeout=10)

    # print(response.text)
    print(response.url,response)
    if response.status_code == 200:
        return response.content
#下载一个关键词的图片
def parse_data(word = '停车场'):
    path = f'/media/chenglei3/77D014CED257D1411/images/bing_pic/{word}/'
    os.makedirs(path,exist_ok=True)
    print(word)
    xuhao = 1
    for page in range(2, 300-300+20):
        response = get_bing_image(word,SFX=page)
        # soups = BeautifulSoup(response, 'lxml')
        # datas = soups.find('ul',class_='dgControl_list').find_all('div', class_='iuscp isv')
        # 使用正则表达式匹配并提取完整的网址，即图片的网址
        url_list = re.findall(r'(?:https?://|\/\/)[\w\d./-]+/th/id/OIP-C\.\w+\?', response)

        print(word,page,len(url_list))
        for url in url_list:
            url = url.rstrip('?')
            OIP =str(url).split('id/')[-1]
            filename = path +str(xuhao)+ '_'  +OIP
            if images_url.findOne({'_id': url, 'status': 'done'}):
                continue #已经下载过的跳过
            reslut = {'_id': url, 'platform': 'bingying', 'keyword': word}
            response = get_pic_data(url) #图片的二进制数据
            if response:
                with open(filename, 'wb') as fd:
                    fd.write(response)
                reslut['status'] = 'done'
            else:
                reslut['status'] = 'fail'
            # 保存数据库
            try:
                images_url.insertOne(reslut)
            except:
                images_url.updateOne({'_id': url}, reslut)
            requests.session().close()
            xuhao += 1

        if len(url_list) <20:
            break # 如果这一页的图片数量少于20，说明已经没有更多图片了，可以退出循环

def main():
    word_20241113 = ['车内', '水杯', '雨伞']
    word_20241113_1 = ['在车内', '在车上', '车后排']
    word_20241114 = ['车内手机', '车内包', '车内办公', '车内钥匙', '车内充电宝', '车内宠物', '车内吃饭','车内打哈欠','车内抽烟','车内吸烟']
    word_20241114_1 = ['查看导航（车内）', '查看行车记录仪（车内）', '车内安排日程', '车内办公', '车内抱着狗', '车内唱歌', '车内吃播',
            '车内吃东西', '车内吃零食', '车内打电话', '车内对话', '车内放松心情', '车内分享趣闻', '车内给孩子讲故事',
            '车内观看视频', '车内嗨', '车内喝水', '车内驾驶位自拍', '车内交流感受', '车内进行冥想', '车内进行思考',
            '车内看电影', '车内看手机', '车内聊天', '车内拍摄风景', '车内派对', '车内清扫', '车内生气', '车内使用平板',
            '车内视频', '车内睡觉', '车内说话', '车内谈论新闻', '车内讨论工作', '车内跳舞', '车内听音乐', '车内痛哭',
            '车内玩游戏', '车内吸烟', '车内享受美食', '车内享受阳光', '车内写作业', '车内欣赏风景', '车内休息',
            '车内娱乐', '车内阅读书籍', '车内照顾孩子', '车内整理物品', '车内准备工作', '车内自嗨', '车内自拍',
            '车上（车内）化妆', '车上（车内）聊天', '车上（车内）睡觉', '抽烟（车内）', '出行自驾车内视频', '出游车内视频',
            '打电话（车内）', '调节空调（车内）', '调整座椅（车内）', '儿童车内', '观察路况（车内）', '行车测试（车内）',
            '进行车内运动', '两口子车上（车内）对话', '清理车内杂物', '设置驾驶模式（车内）', '使用车载充电器（车内）',
            '使用车载屏幕（车内）', '试驾（车内）', '网约车司机视频摆拍（车内）', '系安全带（车内）', '寻找停车位（车内）',
            '孕妇车内', '自驾车内过夜', '自驾车内睡觉']
    word_20241115 = ['狗狗在车内', '猫猫在车内']
    words = list(set(word_20241115))

    # 多进程
    # pool = Pool(processes=min(5,len(words)))
    # pool.map(parse_data, words)
    # pool.close()  # 关闭进程池，不再接受新的进程
    # pool.join()

    # 单进程
    for word in words:
        parse_data(word)

if __name__ == "__main__":
    print('main')
    main()
    # images_url.deleteMany({})

    images_url.close()

